{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Phase 3 Weighted Bagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "\n",
    "import os\n",
    "import re\n",
    "import csv\n",
    "import codecs\n",
    "import gensim\n",
    "import itertools\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import operator\n",
    "import sys\n",
    "\n",
    "from nltk import ngrams\n",
    "from collections import Counter\n",
    "from string import punctuation\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from iwillwin.trainer.supervised_trainer import KerasModelTrainer\n",
    "from iwillwin.data_utils.data_helpers import DataTransformer, DataLoader\n",
    "from iwillwin.config import dataset_config\n",
    "from iwillwin.data_utils.feature_engineering import FeatureCreator\n",
    "\n",
    "from fuzzywuzzy import fuzz\n",
    "from nltk.corpus import stopwords\n",
    "from tqdm import tqdm\n",
    "from scipy.stats import skew, kurtosis\n",
    "from scipy.spatial.distance import cosine, cityblock, jaccard, canberra, euclidean, minkowski, braycurtis\n",
    "from nltk import word_tokenize\n",
    "\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "import os\n",
    "import re\n",
    "import csv\n",
    "import codecs\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import operator\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "\n",
    "########################################\n",
    "## import packages\n",
    "########################################\n",
    "import os\n",
    "import re\n",
    "import csv\n",
    "import codecs\n",
    "import numpy as np\n",
    "np.random.seed(1337)\n",
    "\n",
    "import pandas as pd\n",
    "import operator\n",
    "import sys\n",
    "\n",
    "from string import punctuation\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from iwillwin.trainer.supervised_trainer import KerasModelTrainer\n",
    "from iwillwin.data_utils.data_helpers import DataTransformer, DataLoader\n",
    "from iwillwin.config import dataset_config\n",
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[DataHelper] Apply normalization on value-type columns\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Doing preprocessing...\n",
      "Transforming words to indices...\n",
      "Shape of data tensor: (320552, 50) (320552, 50)\n",
      "Shape of label tensor: (320552,)\n",
      "Preprocessed.\n",
      "Number of unique words 83265\n"
     ]
    }
   ],
   "source": [
    "NB_WORDS, MAX_SEQUENCE_LENGTH = 50000, 50\n",
    "data_transformer = DataTransformer(max_num_words=NB_WORDS, max_sequence_length=MAX_SEQUENCE_LENGTH, char_level=False,\n",
    "                                   normalization=True, features_processed=True)\n",
    "trains_nns, tests_nns, labels = data_transformer.prepare_data(dual=False)\n",
    "print(\"Number of unique words\", len(data_transformer.tokenizer.index_docs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trains_meta = trains_nns[2]\n",
    "tests_meta = tests_nns[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv('../data/dataset/train.csv')\n",
    "test_df = pd.read_csv('../data/dataset/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n"
     ]
    }
   ],
   "source": [
    "rumor_words = ['辟谣', '谣言', '勿传', '假的']\n",
    "\n",
    "def is_rumor(text):\n",
    "    if type(text) != str:\n",
    "        print(text, type(text))\n",
    "        return 0\n",
    "    for rumor_word in rumor_words:\n",
    "        if rumor_word in text:\n",
    "            return 1\n",
    "    return 0\n",
    "\n",
    "def has_split_symbol(text):\n",
    "    if type(text) != str:\n",
    "        return 0\n",
    "    if '|' in text:\n",
    "        return 1\n",
    "    return 0\n",
    "\n",
    "for df in [train_df, test_df]:\n",
    "    df['has_|'] = df['title2_zh'].apply(has_split_symbol)\n",
    "    df['has_rumor_words'] = df['title2_zh'].apply(is_rumor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_has_rumor = train_df.has_rumor_words.values\n",
    "test_has_rumor = test_df.has_rumor_words.values\n",
    "\n",
    "trick_trains_features = np.concatenate((trains_nns[2], train_has_rumor.reshape((-1, 1))), axis=1)\n",
    "trick_tests_features = np.concatenate((tests_nns[2], test_has_rumor.reshape((-1, 1))), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "oof_file_names = sorted([f for f in listdir('../data/p_ensemble/oofs/') if isfile(join('../data/p_ensemble/oofs/', f)) and f != '.gitkeep'])\n",
    "preds_file_names = [name.replace('-Train', '') for name in oof_file_names]\n",
    "\n",
    "oofs = []\n",
    "preds = []\n",
    "for name in oof_file_names:\n",
    "    oofs.append(pd.read_csv('../data/p_ensemble/oofs/' + name))\n",
    "for name in preds_file_names:\n",
    "    preds.append(pd.read_csv('../data/p_ensemble/preds/' + name))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 AddNN-Ensemble-Weighted-StopOnAcc-Train-L0.875371-NB50000.csv\n",
      "1 AddNN-Ensemble-Weighted-StopOnLoss-Train-L0.867090-NB50000.csv\n",
      "2 LightGBM-Ensemble-Train-L0.866327-NB50000.csv\n",
      "3 LightGBMWordLevel-Ensemble-Train-L0.861769-NB50000.csv\n",
      "4 Logistic-Ensemble-Train-L0.866449-NB50000.csv\n",
      "5 Ridge-Ensemble-Train-L0.866237-NB50000.csv\n"
     ]
    }
   ],
   "source": [
    "for i, name in enumerate(oof_file_names):\n",
    "    print(i, name)\n",
    "    \n",
    "trains = pd.DataFrame()\n",
    "tests = pd.DataFrame()\n",
    "\n",
    "for i in range(len(oof_file_names)):\n",
    "    for label_type in ['agreed', 'disagreed', 'unrelated']:\n",
    "        trains['oofs_{}_{}'.format(i, label_type)] = oofs[i][label_type].values\n",
    "        tests['oofs_pred{}_{}'.format(i, label_type)] = preds[i][label_type].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "unrelated = pd.DataFrame()\n",
    "agreeds = pd.DataFrame()\n",
    "disagreeds = pd.DataFrame()\n",
    "\n",
    "#check_oofs = True\n",
    "check_oofs = False\n",
    "\n",
    "\n",
    "if check_oofs:\n",
    "    for i, oof in enumerate(oofs):\n",
    "        agreeds['oofs_agreed_{}'.format(i)] = oofs[i]['agreed'].values\n",
    "        unrelated['oofs_unrelated_{}'.format(i)] = oofs[i]['unrelated'].values\n",
    "        disagreeds['oofs_disagreeds_{}'.format(i)] = oofs[i]['disagreed'].values\n",
    "else:\n",
    "    for i, oof in enumerate(oofs):\n",
    "        agreeds['oofs_agreed_{}'.format(i)] = preds[i]['agreed'].values\n",
    "        unrelated['oofs_unrelated_{}'.format(i)] = preds[i]['unrelated'].values\n",
    "        disagreeds['oofs_disagreeds_{}'.format(i)] = preds[i]['disagreed'].values  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oofs_agreed_0</th>\n",
       "      <th>oofs_agreed_1</th>\n",
       "      <th>oofs_agreed_2</th>\n",
       "      <th>oofs_agreed_3</th>\n",
       "      <th>oofs_agreed_4</th>\n",
       "      <th>oofs_agreed_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999986</td>\n",
       "      <td>0.999710</td>\n",
       "      <td>0.997443</td>\n",
       "      <td>0.994811</td>\n",
       "      <td>0.998455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_1</th>\n",
       "      <td>0.999986</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999740</td>\n",
       "      <td>0.997455</td>\n",
       "      <td>0.994560</td>\n",
       "      <td>0.998365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_2</th>\n",
       "      <td>0.999710</td>\n",
       "      <td>0.999740</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.997434</td>\n",
       "      <td>0.993696</td>\n",
       "      <td>0.998148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_3</th>\n",
       "      <td>0.997443</td>\n",
       "      <td>0.997455</td>\n",
       "      <td>0.997434</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.989557</td>\n",
       "      <td>0.995106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_4</th>\n",
       "      <td>0.994811</td>\n",
       "      <td>0.994560</td>\n",
       "      <td>0.993696</td>\n",
       "      <td>0.989557</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.995949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_5</th>\n",
       "      <td>0.998455</td>\n",
       "      <td>0.998365</td>\n",
       "      <td>0.998148</td>\n",
       "      <td>0.995106</td>\n",
       "      <td>0.995949</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               oofs_agreed_0  oofs_agreed_1  oofs_agreed_2  oofs_agreed_3  \\\n",
       "oofs_agreed_0       1.000000       0.999986       0.999710       0.997443   \n",
       "oofs_agreed_1       0.999986       1.000000       0.999740       0.997455   \n",
       "oofs_agreed_2       0.999710       0.999740       1.000000       0.997434   \n",
       "oofs_agreed_3       0.997443       0.997455       0.997434       1.000000   \n",
       "oofs_agreed_4       0.994811       0.994560       0.993696       0.989557   \n",
       "oofs_agreed_5       0.998455       0.998365       0.998148       0.995106   \n",
       "\n",
       "               oofs_agreed_4  oofs_agreed_5  \n",
       "oofs_agreed_0       0.994811       0.998455  \n",
       "oofs_agreed_1       0.994560       0.998365  \n",
       "oofs_agreed_2       0.993696       0.998148  \n",
       "oofs_agreed_3       0.989557       0.995106  \n",
       "oofs_agreed_4       1.000000       0.995949  \n",
       "oofs_agreed_5       0.995949       1.000000  "
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agreeds.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oofs_disagreeds_0</th>\n",
       "      <th>oofs_disagreeds_1</th>\n",
       "      <th>oofs_disagreeds_2</th>\n",
       "      <th>oofs_disagreeds_3</th>\n",
       "      <th>oofs_disagreeds_4</th>\n",
       "      <th>oofs_disagreeds_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.976072</td>\n",
       "      <td>0.980943</td>\n",
       "      <td>0.977130</td>\n",
       "      <td>0.911610</td>\n",
       "      <td>0.920323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_1</th>\n",
       "      <td>0.976072</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.997218</td>\n",
       "      <td>0.994303</td>\n",
       "      <td>0.974962</td>\n",
       "      <td>0.962633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_2</th>\n",
       "      <td>0.980943</td>\n",
       "      <td>0.997218</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.995663</td>\n",
       "      <td>0.968548</td>\n",
       "      <td>0.956897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_3</th>\n",
       "      <td>0.977130</td>\n",
       "      <td>0.994303</td>\n",
       "      <td>0.995663</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.964323</td>\n",
       "      <td>0.953642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_4</th>\n",
       "      <td>0.911610</td>\n",
       "      <td>0.974962</td>\n",
       "      <td>0.968548</td>\n",
       "      <td>0.964323</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.950559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_5</th>\n",
       "      <td>0.920323</td>\n",
       "      <td>0.962633</td>\n",
       "      <td>0.956897</td>\n",
       "      <td>0.953642</td>\n",
       "      <td>0.950559</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   oofs_disagreeds_0  oofs_disagreeds_1  oofs_disagreeds_2  \\\n",
       "oofs_disagreeds_0           1.000000           0.976072           0.980943   \n",
       "oofs_disagreeds_1           0.976072           1.000000           0.997218   \n",
       "oofs_disagreeds_2           0.980943           0.997218           1.000000   \n",
       "oofs_disagreeds_3           0.977130           0.994303           0.995663   \n",
       "oofs_disagreeds_4           0.911610           0.974962           0.968548   \n",
       "oofs_disagreeds_5           0.920323           0.962633           0.956897   \n",
       "\n",
       "                   oofs_disagreeds_3  oofs_disagreeds_4  oofs_disagreeds_5  \n",
       "oofs_disagreeds_0           0.977130           0.911610           0.920323  \n",
       "oofs_disagreeds_1           0.994303           0.974962           0.962633  \n",
       "oofs_disagreeds_2           0.995663           0.968548           0.956897  \n",
       "oofs_disagreeds_3           1.000000           0.964323           0.953642  \n",
       "oofs_disagreeds_4           0.964323           1.000000           0.950559  \n",
       "oofs_disagreeds_5           0.953642           0.950559           1.000000  "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "disagreeds.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oofs_unrelated_0</th>\n",
       "      <th>oofs_unrelated_1</th>\n",
       "      <th>oofs_unrelated_2</th>\n",
       "      <th>oofs_unrelated_3</th>\n",
       "      <th>oofs_unrelated_4</th>\n",
       "      <th>oofs_unrelated_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.992047</td>\n",
       "      <td>0.992922</td>\n",
       "      <td>0.989629</td>\n",
       "      <td>0.980826</td>\n",
       "      <td>0.992267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_1</th>\n",
       "      <td>0.992047</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999537</td>\n",
       "      <td>0.997011</td>\n",
       "      <td>0.992254</td>\n",
       "      <td>0.999502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_2</th>\n",
       "      <td>0.992922</td>\n",
       "      <td>0.999537</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.996987</td>\n",
       "      <td>0.990938</td>\n",
       "      <td>0.999046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_3</th>\n",
       "      <td>0.989629</td>\n",
       "      <td>0.997011</td>\n",
       "      <td>0.996987</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.986483</td>\n",
       "      <td>0.996372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_4</th>\n",
       "      <td>0.980826</td>\n",
       "      <td>0.992254</td>\n",
       "      <td>0.990938</td>\n",
       "      <td>0.986483</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.991760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_5</th>\n",
       "      <td>0.992267</td>\n",
       "      <td>0.999502</td>\n",
       "      <td>0.999046</td>\n",
       "      <td>0.996372</td>\n",
       "      <td>0.991760</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  oofs_unrelated_0  oofs_unrelated_1  oofs_unrelated_2  \\\n",
       "oofs_unrelated_0          1.000000          0.992047          0.992922   \n",
       "oofs_unrelated_1          0.992047          1.000000          0.999537   \n",
       "oofs_unrelated_2          0.992922          0.999537          1.000000   \n",
       "oofs_unrelated_3          0.989629          0.997011          0.996987   \n",
       "oofs_unrelated_4          0.980826          0.992254          0.990938   \n",
       "oofs_unrelated_5          0.992267          0.999502          0.999046   \n",
       "\n",
       "                  oofs_unrelated_3  oofs_unrelated_4  oofs_unrelated_5  \n",
       "oofs_unrelated_0          0.989629          0.980826          0.992267  \n",
       "oofs_unrelated_1          0.997011          0.992254          0.999502  \n",
       "oofs_unrelated_2          0.996987          0.990938          0.999046  \n",
       "oofs_unrelated_3          1.000000          0.986483          0.996372  \n",
       "oofs_unrelated_4          0.986483          1.000000          0.991760  \n",
       "oofs_unrelated_5          0.996372          0.991760          1.000000  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unrelated.corr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare Different Inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Only use oofs\n",
    "ensemble_trains = trains.values\n",
    "ensemble_tests = tests.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble With NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "########################################\n",
    "## import packages\n",
    "########################################\n",
    "import os\n",
    "import re\n",
    "import csv\n",
    "import codecs\n",
    "import numpy as np\n",
    "np.random.seed(1337)\n",
    "\n",
    "import pandas as pd\n",
    "import operator\n",
    "import sys\n",
    "\n",
    "from string import punctuation\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from iwillwin.trainer.supervised_trainer import KerasModelTrainer\n",
    "from iwillwin.data_utils.data_helpers import DataTransformer, DataLoader\n",
    "from iwillwin.model.sim_zoos import *\n",
    "import tensorflow as tf\n",
    "from keras.layers import Dense, Input, MaxPooling1D, CuDNNLSTM, Embedding, Add, Lambda, Dropout, Activation, SpatialDropout1D, Reshape, GlobalAveragePooling1D, merge, Flatten, Bidirectional, CuDNNGRU, add, Conv1D, GlobalMaxPooling1D\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.models import Model\n",
    "from keras import optimizers\n",
    "from keras import initializers\n",
    "from keras.engine import InputSpec, Layer\n",
    "from iwillwin.config import dataset_config, model_config\n",
    "from keras.models import Sequential\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers.core import Lambda, Dense, Dropout\n",
    "from keras.layers.recurrent import LSTM, GRU\n",
    "from keras.layers.wrappers import Bidirectional\n",
    "from keras.legacy.layers import Highway\n",
    "from keras.layers import TimeDistributed\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "import keras.backend as K\n",
    "\n",
    "from sklearn.metrics import roc_auc_score, log_loss\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from sklearn.metrics import log_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras import regularizers\n",
    "\n",
    "def weighted_accuracy(y_true, y_pred):\n",
    "    # hard version, 0.5 counts as correct\n",
    "    weight = np.array([[1/16, 1/15, 1/5]])\n",
    "    norm = [(1/16) + (1/15) + (1/5)]\n",
    "    weight_mask = weight * y_true\n",
    "    label_weights = K.argmax(weight_mask, axis=-1)\n",
    "    \n",
    "    true_label = K.argmax(y_true, axis=-1)\n",
    "    pred_label = K.argmax(y_pred, axis=-1)\n",
    "    \n",
    "    res = K.cast(K.equal(true_label, pred_label), tf.int64) * label_weights / K.sum(label_weights)\n",
    "    res = K.sum(res)\n",
    "    return res\n",
    "\n",
    "def get_dense_add_net(feature_nums):\n",
    "    features_inputs = Input(shape=(feature_nums,), name='mata-features', dtype=\"float32\")\n",
    "    features = features_inputs\n",
    "    \n",
    "    depth = 5\n",
    "    for i in range(depth):\n",
    "        new_features = Dense(24, activation='relu')(features)\n",
    "        new_features = Dropout(0.1)(new_features)\n",
    "        features = Concatenate()([features, new_features])\n",
    "\n",
    "    h = Highway(activation='relu')(features)\n",
    "    out_ = Dense(3, activation='softmax')(h)\n",
    "    \n",
    "    model = Model(inputs=[features_inputs], outputs=out_)\n",
    "    model.compile(optimizer=Adam(lr=1e-3, decay=1e-6,), loss='categorical_crossentropy',\n",
    "    metrics=['accuracy', weighted_accuracy])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_logit_net(feature_nums):\n",
    "    features_inputs = Input(shape=(feature_nums,), name='mata-features', dtype=\"float32\")    \n",
    "    out_ = Dense(3, activation='softmax')(features_inputs)\n",
    "    model = Model(inputs=[features_inputs], outputs=out_)\n",
    "    model.compile(optimizer=Adam(lr=1e-3, decay=1e-6,), loss='categorical_crossentropy',\n",
    "    metrics=['accuracy', weighted_accuracy])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def _agent_get_model():\n",
    "    return get_dense_add_net(ensemble_trains.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def np_weighted_accuracy(y_true, y_pred):\n",
    "    weight = np.array([[1/16, 1/15, 1/5]])\n",
    "    norm = [(1/16) + (1/15) + (1/5)]\n",
    "    weight_mask = weight * y_true\n",
    "    weight_mask = np.max(weight_mask, axis=-1)\n",
    "    norms = np.sum(weight_mask)\n",
    "    \n",
    "    y_true = np.argmax(y_true, axis=-1)\n",
    "    y_pred = np.argmax(y_pred, axis=-1)\n",
    "    \n",
    "    res = ((y_true == y_pred) * weight_mask).sum() / norms\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weighted_accuracy(y_true, y_pred):\n",
    "    weight = np.array([[1/16, 1/15, 1/5]])\n",
    "    norm = [(1/16) + (1/15) + (1/5)]\n",
    "    weight_mask = weight * y_true\n",
    "    \n",
    "    y_pred = K.cast(y_pred > 0.5, 'int32') # Hard\n",
    "    y_true = K.cast(y_true, 'int32')\n",
    "    \n",
    "    res = K.cast(K.equal(y_pred, y_true), 'float32') * weight_mask / K.sum(weight_mask)\n",
    "    res = K.sum(res)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weighted_accuracy(y_true, y_pred):\n",
    "    weight = np.array([[1/16, 1/15, 1/5]])\n",
    "    norm = [(1/16) + (1/15) + (1/5)]\n",
    "    weight_mask = weight * y_true\n",
    "    label_weights = K.max(K.cast(weight_mask, 'float32'), axis=-1)\n",
    "    \n",
    "    true_label = K.argmax(y_true, axis=-1)\n",
    "    pred_label = K.argmax(y_pred, axis=-1)\n",
    "    \n",
    "    res = K.cast(K.equal(true_label, pred_label), tf.float32) * label_weights / K.sum(label_weights)\n",
    "    res = K.sum(res)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import importlib\n",
    "\n",
    "from sklearn.metrics import roc_auc_score, log_loss\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "from iwillwin.config import model_config\n",
    "\n",
    "class ModelTrainer(object):\n",
    "\n",
    "    def __init__(self, model_stamp, epoch_num, learning_rate=1e-3,\n",
    "                 shuffle_inputs=False, verbose_round=40, early_stopping_round=8):\n",
    "        self.models = []\n",
    "        self.model_stamp = model_stamp\n",
    "        self.val_loss = -1\n",
    "        self.auc = -1\n",
    "        self.epoch_num = epoch_num\n",
    "        self.learning_rate = learning_rate\n",
    "        self.eps = 1e-10\n",
    "        self.verbose_round = verbose_round\n",
    "        self.early_stopping_round = early_stopping_round\n",
    "        self.shuffle_inputs = shuffle_inputs\n",
    "        self.class_weight = [0.93, 1.21]\n",
    "\n",
    "    def train_folds(self, features, y, fold_count, batch_size, get_model_func, augments=None, skip_fold=0, patience=10, scale_sample_weight=False,\n",
    "                    class_weight=None, self_aware=False, swap_input=False):\n",
    "        weight_val=scale_sample_weight\n",
    "        class_weight=None\n",
    "        fold_size = len(features) // fold_count\n",
    "        models = []\n",
    "        fold_predictions = []\n",
    "        score = 0\n",
    "\n",
    "        for fold_id in range(0, fold_count):\n",
    "            fold_start = fold_size * fold_id\n",
    "            fold_end = fold_start + fold_size\n",
    "\n",
    "            if fold_id == fold_count - 1:\n",
    "                fold_end = len(features)\n",
    "\n",
    "            train_features = np.concatenate([features[:fold_start], features[fold_end:]])\n",
    "            train_y = np.concatenate([y[:fold_start], y[fold_end:]])\n",
    "            \n",
    "            val_features = features[fold_start:fold_end]\n",
    "            val_y = y[fold_start:fold_end]\n",
    "            fold_pos = (np.sum(train_y) / len(train_features))\n",
    "\n",
    "            train_data = {\n",
    "                \"mata-features\": train_features,\n",
    "            }\n",
    "\n",
    "            val_data = {\n",
    "                \"mata-features\": val_features,\n",
    "            }\n",
    "\n",
    "            model, bst_val_score, fold_prediction = self._train_model_by_logloss(\n",
    "                get_model_func(), batch_size, train_data, train_y, val_data, val_y, fold_id, patience, class_weight, weight_val=None)\n",
    "    \n",
    "            score += bst_val_score\n",
    "            models.append(model)\n",
    "            fold_predictions.append(fold_prediction)\n",
    "\n",
    "        self.models = models\n",
    "        self.val_loss = score / fold_count\n",
    "        return models, self.val_loss, fold_predictions\n",
    "\n",
    "    def _train_model_by_logloss(self, model, batch_size, train_x, train_y, val_x, val_y, fold_id, patience):\n",
    "        # return a list which holds [models, val_loss, auc, prediction]\n",
    "        raise NotImplementedError\n",
    "\n",
    "class KerasModelTrainer(ModelTrainer):\n",
    "\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        super(KerasModelTrainer, self).__init__(*args, **kwargs)\n",
    "        pass\n",
    "\n",
    "    def _train_model_by_logloss(self, model, batch_size, train_x, train_y, val_x, val_y, fold_id, patience, class_weight, weight_val):\n",
    "        early_stopping = EarlyStopping(monitor='val_loss', patience=10)\n",
    "        bst_model_path = self.model_stamp + str(fold_id) + '.h5'\n",
    "        val_data = (val_x, val_y, weight_val) if weight_val is not None else (val_x, val_y)\n",
    "        model_checkpoint = ModelCheckpoint(bst_model_path, save_best_only=True, save_weights_only=True)\n",
    "        hist = model.fit(train_x, train_y,\n",
    "                         validation_data=val_data,\n",
    "                         epochs=self.epoch_num, batch_size=batch_size, shuffle=True,\n",
    "                         verbose=2,\n",
    "                         class_weight={0: 1/16, 1: 1/15, 2: 1/5},\n",
    "                         callbacks=[early_stopping, model_checkpoint],)\n",
    "        bst_val_score = max(hist.history['val_weighted_accuracy'])\n",
    "        model.load_weights(bst_model_path)\n",
    "        predictions = model.predict(val_x)\n",
    "\n",
    "        return model, bst_val_score, predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\keras\\legacy\\layers.py:198: UserWarning: The `Highway` layer is deprecated and will be removed after 06/2017.\n",
      "  warnings.warn('The `Highway` layer is deprecated '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 18)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_145 (Dense)               (None, 24)           456         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_121 (Dropout)           (None, 24)           0           dense_145[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_121 (Concatenate)   (None, 42)           0           mata-features[0][0]              \n",
      "                                                                 dropout_121[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_146 (Dense)               (None, 24)           1032        concatenate_121[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_122 (Dropout)           (None, 24)           0           dense_146[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_122 (Concatenate)   (None, 66)           0           concatenate_121[0][0]            \n",
      "                                                                 dropout_122[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_147 (Dense)               (None, 24)           1608        concatenate_122[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_123 (Dropout)           (None, 24)           0           dense_147[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_123 (Concatenate)   (None, 90)           0           concatenate_122[0][0]            \n",
      "                                                                 dropout_123[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_148 (Dense)               (None, 24)           2184        concatenate_123[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_124 (Dropout)           (None, 24)           0           dense_148[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_124 (Concatenate)   (None, 114)          0           concatenate_123[0][0]            \n",
      "                                                                 dropout_124[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_149 (Dense)               (None, 24)           2760        concatenate_124[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_125 (Dropout)           (None, 24)           0           dense_149[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_125 (Concatenate)   (None, 138)          0           concatenate_124[0][0]            \n",
      "                                                                 dropout_125[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "highway_25 (Highway)            (None, 138)          38364       concatenate_125[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_150 (Dense)               (None, 3)            417         highway_25[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 46,821\n",
      "Trainable params: 46,821\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 3s - loss: 0.0217 - acc: 0.8771 - weighted_accuracy: 0.8668 - val_loss: 0.2721 - val_acc: 0.8849 - val_weighted_accuracy: 0.8768\n",
      "Epoch 2/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8835 - weighted_accuracy: 0.8751 - val_loss: 0.2678 - val_acc: 0.8859 - val_weighted_accuracy: 0.8769\n",
      "Epoch 3/500\n",
      " - 2s - loss: 0.0193 - acc: 0.8838 - weighted_accuracy: 0.8755 - val_loss: 0.2686 - val_acc: 0.8841 - val_weighted_accuracy: 0.8758\n",
      "Epoch 4/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8838 - weighted_accuracy: 0.8757 - val_loss: 0.2638 - val_acc: 0.8857 - val_weighted_accuracy: 0.8770\n",
      "Epoch 5/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8836 - weighted_accuracy: 0.8756 - val_loss: 0.2684 - val_acc: 0.8849 - val_weighted_accuracy: 0.8768\n",
      "Epoch 6/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8837 - weighted_accuracy: 0.8757 - val_loss: 0.2642 - val_acc: 0.8854 - val_weighted_accuracy: 0.8769\n",
      "Epoch 7/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8839 - weighted_accuracy: 0.8757 - val_loss: 0.2657 - val_acc: 0.8863 - val_weighted_accuracy: 0.8772\n",
      "Epoch 8/500\n",
      " - 1s - loss: 0.0191 - acc: 0.8837 - weighted_accuracy: 0.8757 - val_loss: 0.2743 - val_acc: 0.8822 - val_weighted_accuracy: 0.8757\n",
      "Epoch 9/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8840 - weighted_accuracy: 0.8757 - val_loss: 0.2693 - val_acc: 0.8830 - val_weighted_accuracy: 0.8758\n",
      "Epoch 10/500\n",
      " - 1s - loss: 0.0191 - acc: 0.8839 - weighted_accuracy: 0.8756 - val_loss: 0.2665 - val_acc: 0.8856 - val_weighted_accuracy: 0.8768\n",
      "Epoch 11/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8839 - weighted_accuracy: 0.8759 - val_loss: 0.2634 - val_acc: 0.8862 - val_weighted_accuracy: 0.8773\n",
      "Epoch 12/500\n",
      " - 1s - loss: 0.0191 - acc: 0.8837 - weighted_accuracy: 0.8756 - val_loss: 0.2645 - val_acc: 0.8856 - val_weighted_accuracy: 0.8769\n",
      "Epoch 13/500\n",
      " - 1s - loss: 0.0191 - acc: 0.8840 - weighted_accuracy: 0.8758 - val_loss: 0.2698 - val_acc: 0.8851 - val_weighted_accuracy: 0.8766\n",
      "Epoch 14/500\n",
      " - 1s - loss: 0.0191 - acc: 0.8841 - weighted_accuracy: 0.8758 - val_loss: 0.2688 - val_acc: 0.8833 - val_weighted_accuracy: 0.8760\n",
      "Epoch 15/500\n",
      " - 1s - loss: 0.0191 - acc: 0.8837 - weighted_accuracy: 0.8758 - val_loss: 0.2692 - val_acc: 0.8844 - val_weighted_accuracy: 0.8767\n",
      "Epoch 16/500\n",
      " - 1s - loss: 0.0190 - acc: 0.8840 - weighted_accuracy: 0.8759 - val_loss: 0.2699 - val_acc: 0.8818 - val_weighted_accuracy: 0.8755\n",
      "Epoch 17/500\n",
      " - 1s - loss: 0.0190 - acc: 0.8840 - weighted_accuracy: 0.8761 - val_loss: 0.2645 - val_acc: 0.8857 - val_weighted_accuracy: 0.8769\n",
      "Epoch 18/500\n",
      " - 1s - loss: 0.0190 - acc: 0.8844 - weighted_accuracy: 0.8762 - val_loss: 0.2669 - val_acc: 0.8842 - val_weighted_accuracy: 0.8757\n",
      "Epoch 19/500\n",
      " - 1s - loss: 0.0190 - acc: 0.8841 - weighted_accuracy: 0.8759 - val_loss: 0.2689 - val_acc: 0.8834 - val_weighted_accuracy: 0.8761\n",
      "Epoch 20/500\n",
      " - 1s - loss: 0.0190 - acc: 0.8840 - weighted_accuracy: 0.8760 - val_loss: 0.2667 - val_acc: 0.8846 - val_weighted_accuracy: 0.8768\n",
      "Epoch 21/500\n",
      " - 1s - loss: 0.0190 - acc: 0.8841 - weighted_accuracy: 0.8759 - val_loss: 0.2691 - val_acc: 0.8828 - val_weighted_accuracy: 0.8758\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 18)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_151 (Dense)               (None, 24)           456         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_126 (Dropout)           (None, 24)           0           dense_151[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_126 (Concatenate)   (None, 42)           0           mata-features[0][0]              \n",
      "                                                                 dropout_126[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_152 (Dense)               (None, 24)           1032        concatenate_126[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_127 (Dropout)           (None, 24)           0           dense_152[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_127 (Concatenate)   (None, 66)           0           concatenate_126[0][0]            \n",
      "                                                                 dropout_127[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_153 (Dense)               (None, 24)           1608        concatenate_127[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_128 (Dropout)           (None, 24)           0           dense_153[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_128 (Concatenate)   (None, 90)           0           concatenate_127[0][0]            \n",
      "                                                                 dropout_128[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_154 (Dense)               (None, 24)           2184        concatenate_128[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_129 (Dropout)           (None, 24)           0           dense_154[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_129 (Concatenate)   (None, 114)          0           concatenate_128[0][0]            \n",
      "                                                                 dropout_129[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_155 (Dense)               (None, 24)           2760        concatenate_129[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_130 (Dropout)           (None, 24)           0           dense_155[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_130 (Concatenate)   (None, 138)          0           concatenate_129[0][0]            \n",
      "                                                                 dropout_130[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "highway_26 (Highway)            (None, 138)          38364       concatenate_130[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_156 (Dense)               (None, 3)            417         highway_26[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 46,821\n",
      "Trainable params: 46,821\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 3s - loss: 0.0219 - acc: 0.8778 - weighted_accuracy: 0.8672 - val_loss: 0.2452 - val_acc: 0.8923 - val_weighted_accuracy: 0.8845\n",
      "Epoch 2/500\n",
      " - 1s - loss: 0.0197 - acc: 0.8831 - weighted_accuracy: 0.8743 - val_loss: 0.2387 - val_acc: 0.8944 - val_weighted_accuracy: 0.8856\n",
      "Epoch 3/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8829 - weighted_accuracy: 0.8745 - val_loss: 0.2403 - val_acc: 0.8933 - val_weighted_accuracy: 0.8853\n",
      "Epoch 4/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8830 - weighted_accuracy: 0.8744 - val_loss: 0.2407 - val_acc: 0.8922 - val_weighted_accuracy: 0.8846\n",
      "Epoch 5/500\n",
      " - 1s - loss: 0.0194 - acc: 0.8830 - weighted_accuracy: 0.8751 - val_loss: 0.2371 - val_acc: 0.8943 - val_weighted_accuracy: 0.8851\n",
      "Epoch 6/500\n",
      " - 1s - loss: 0.0194 - acc: 0.8831 - weighted_accuracy: 0.8746 - val_loss: 0.2364 - val_acc: 0.8943 - val_weighted_accuracy: 0.8853\n",
      "Epoch 7/500\n",
      " - 1s - loss: 0.0194 - acc: 0.8830 - weighted_accuracy: 0.8747 - val_loss: 0.2380 - val_acc: 0.8924 - val_weighted_accuracy: 0.8846\n",
      "Epoch 8/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8831 - weighted_accuracy: 0.8750 - val_loss: 0.2420 - val_acc: 0.8920 - val_weighted_accuracy: 0.8845\n",
      "Epoch 9/500\n",
      " - 1s - loss: 0.0194 - acc: 0.8827 - weighted_accuracy: 0.8746 - val_loss: 0.2374 - val_acc: 0.8941 - val_weighted_accuracy: 0.8850\n",
      "Epoch 10/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8831 - weighted_accuracy: 0.8748 - val_loss: 0.2367 - val_acc: 0.8945 - val_weighted_accuracy: 0.8852\n",
      "Epoch 11/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8833 - weighted_accuracy: 0.8751 - val_loss: 0.2379 - val_acc: 0.8937 - val_weighted_accuracy: 0.8853\n",
      "Epoch 12/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8830 - weighted_accuracy: 0.8748 - val_loss: 0.2383 - val_acc: 0.8937 - val_weighted_accuracy: 0.8854\n",
      "Epoch 13/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8832 - weighted_accuracy: 0.8749 - val_loss: 0.2359 - val_acc: 0.8949 - val_weighted_accuracy: 0.8856\n",
      "Epoch 14/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8829 - weighted_accuracy: 0.8748 - val_loss: 0.2392 - val_acc: 0.8924 - val_weighted_accuracy: 0.8846\n",
      "Epoch 15/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8831 - weighted_accuracy: 0.8748 - val_loss: 0.2363 - val_acc: 0.8950 - val_weighted_accuracy: 0.8858\n",
      "Epoch 16/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8831 - weighted_accuracy: 0.8750 - val_loss: 0.2391 - val_acc: 0.8929 - val_weighted_accuracy: 0.8847\n",
      "Epoch 17/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8832 - weighted_accuracy: 0.8752 - val_loss: 0.2359 - val_acc: 0.8956 - val_weighted_accuracy: 0.8857\n",
      "Epoch 18/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8831 - weighted_accuracy: 0.8748 - val_loss: 0.2400 - val_acc: 0.8941 - val_weighted_accuracy: 0.8855\n",
      "Epoch 19/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8833 - weighted_accuracy: 0.8751 - val_loss: 0.2376 - val_acc: 0.8948 - val_weighted_accuracy: 0.8856\n",
      "Epoch 20/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8831 - weighted_accuracy: 0.8749 - val_loss: 0.2354 - val_acc: 0.8955 - val_weighted_accuracy: 0.8858\n",
      "Epoch 21/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8833 - weighted_accuracy: 0.8752 - val_loss: 0.2351 - val_acc: 0.8954 - val_weighted_accuracy: 0.8858\n",
      "Epoch 22/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8835 - weighted_accuracy: 0.8752 - val_loss: 0.2371 - val_acc: 0.8949 - val_weighted_accuracy: 0.8861\n",
      "Epoch 23/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8833 - weighted_accuracy: 0.8749 - val_loss: 0.2379 - val_acc: 0.8951 - val_weighted_accuracy: 0.8857\n",
      "Epoch 24/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8834 - weighted_accuracy: 0.8750 - val_loss: 0.2380 - val_acc: 0.8948 - val_weighted_accuracy: 0.8855\n",
      "Epoch 25/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8833 - weighted_accuracy: 0.8751 - val_loss: 0.2385 - val_acc: 0.8946 - val_weighted_accuracy: 0.8856\n",
      "Epoch 26/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8831 - weighted_accuracy: 0.8751 - val_loss: 0.2371 - val_acc: 0.8958 - val_weighted_accuracy: 0.8858\n",
      "Epoch 27/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8834 - weighted_accuracy: 0.8751 - val_loss: 0.2354 - val_acc: 0.8948 - val_weighted_accuracy: 0.8853\n",
      "Epoch 28/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8835 - weighted_accuracy: 0.8753 - val_loss: 0.2360 - val_acc: 0.8952 - val_weighted_accuracy: 0.8856\n",
      "Epoch 29/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8834 - weighted_accuracy: 0.8750 - val_loss: 0.2375 - val_acc: 0.8938 - val_weighted_accuracy: 0.8854\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 30/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8831 - weighted_accuracy: 0.8750 - val_loss: 0.2376 - val_acc: 0.8957 - val_weighted_accuracy: 0.8857\n",
      "Epoch 31/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8833 - weighted_accuracy: 0.8750 - val_loss: 0.2337 - val_acc: 0.8963 - val_weighted_accuracy: 0.8858\n",
      "Epoch 32/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8833 - weighted_accuracy: 0.8750 - val_loss: 0.2393 - val_acc: 0.8938 - val_weighted_accuracy: 0.8853\n",
      "Epoch 33/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8836 - weighted_accuracy: 0.8753 - val_loss: 0.2338 - val_acc: 0.8964 - val_weighted_accuracy: 0.8856\n",
      "Epoch 34/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8833 - weighted_accuracy: 0.8751 - val_loss: 0.2364 - val_acc: 0.8952 - val_weighted_accuracy: 0.8853\n",
      "Epoch 35/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8833 - weighted_accuracy: 0.8750 - val_loss: 0.2353 - val_acc: 0.8952 - val_weighted_accuracy: 0.8854\n",
      "Epoch 36/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8835 - weighted_accuracy: 0.8751 - val_loss: 0.2344 - val_acc: 0.8957 - val_weighted_accuracy: 0.8857\n",
      "Epoch 37/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8836 - weighted_accuracy: 0.8754 - val_loss: 0.2369 - val_acc: 0.8947 - val_weighted_accuracy: 0.8854\n",
      "Epoch 38/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8834 - weighted_accuracy: 0.8753 - val_loss: 0.2350 - val_acc: 0.8964 - val_weighted_accuracy: 0.8860\n",
      "Epoch 39/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8835 - weighted_accuracy: 0.8752 - val_loss: 0.2347 - val_acc: 0.8957 - val_weighted_accuracy: 0.8856\n",
      "Epoch 40/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8833 - weighted_accuracy: 0.8753 - val_loss: 0.2379 - val_acc: 0.8949 - val_weighted_accuracy: 0.8855\n",
      "Epoch 41/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8832 - weighted_accuracy: 0.8750 - val_loss: 0.2359 - val_acc: 0.8950 - val_weighted_accuracy: 0.8849\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 18)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_157 (Dense)               (None, 24)           456         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_131 (Dropout)           (None, 24)           0           dense_157[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_131 (Concatenate)   (None, 42)           0           mata-features[0][0]              \n",
      "                                                                 dropout_131[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_158 (Dense)               (None, 24)           1032        concatenate_131[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_132 (Dropout)           (None, 24)           0           dense_158[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_132 (Concatenate)   (None, 66)           0           concatenate_131[0][0]            \n",
      "                                                                 dropout_132[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_159 (Dense)               (None, 24)           1608        concatenate_132[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_133 (Dropout)           (None, 24)           0           dense_159[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_133 (Concatenate)   (None, 90)           0           concatenate_132[0][0]            \n",
      "                                                                 dropout_133[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_160 (Dense)               (None, 24)           2184        concatenate_133[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_134 (Dropout)           (None, 24)           0           dense_160[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_134 (Concatenate)   (None, 114)          0           concatenate_133[0][0]            \n",
      "                                                                 dropout_134[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_161 (Dense)               (None, 24)           2760        concatenate_134[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_135 (Dropout)           (None, 24)           0           dense_161[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_135 (Concatenate)   (None, 138)          0           concatenate_134[0][0]            \n",
      "                                                                 dropout_135[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "highway_27 (Highway)            (None, 138)          38364       concatenate_135[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_162 (Dense)               (None, 3)            417         highway_27[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 46,821\n",
      "Trainable params: 46,821\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 3s - loss: 0.0216 - acc: 0.8791 - weighted_accuracy: 0.8683 - val_loss: 0.2682 - val_acc: 0.8814 - val_weighted_accuracy: 0.8740\n",
      "Epoch 2/500\n",
      " - 1s - loss: 0.0195 - acc: 0.8836 - weighted_accuracy: 0.8752 - val_loss: 0.2652 - val_acc: 0.8829 - val_weighted_accuracy: 0.8746\n",
      "Epoch 3/500\n",
      " - 1s - loss: 0.0194 - acc: 0.8839 - weighted_accuracy: 0.8756 - val_loss: 0.2677 - val_acc: 0.8810 - val_weighted_accuracy: 0.8736\n",
      "Epoch 4/500\n",
      " - 2s - loss: 0.0193 - acc: 0.8841 - weighted_accuracy: 0.8759 - val_loss: 0.2658 - val_acc: 0.8826 - val_weighted_accuracy: 0.8742\n",
      "Epoch 5/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8839 - weighted_accuracy: 0.8756 - val_loss: 0.2636 - val_acc: 0.8839 - val_weighted_accuracy: 0.8751\n",
      "Epoch 6/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8839 - weighted_accuracy: 0.8755 - val_loss: 0.2656 - val_acc: 0.8821 - val_weighted_accuracy: 0.8740\n",
      "Epoch 7/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8841 - weighted_accuracy: 0.8759 - val_loss: 0.2612 - val_acc: 0.8835 - val_weighted_accuracy: 0.8751\n",
      "Epoch 8/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8843 - weighted_accuracy: 0.8759 - val_loss: 0.2659 - val_acc: 0.8811 - val_weighted_accuracy: 0.8739\n",
      "Epoch 9/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8839 - weighted_accuracy: 0.8756 - val_loss: 0.2627 - val_acc: 0.8830 - val_weighted_accuracy: 0.8743\n",
      "Epoch 10/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8846 - weighted_accuracy: 0.8761 - val_loss: 0.2649 - val_acc: 0.8816 - val_weighted_accuracy: 0.8743\n",
      "Epoch 11/500\n",
      " - 1s - loss: 0.0191 - acc: 0.8843 - weighted_accuracy: 0.8758 - val_loss: 0.2584 - val_acc: 0.8855 - val_weighted_accuracy: 0.8761\n",
      "Epoch 12/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8844 - weighted_accuracy: 0.8761 - val_loss: 0.2622 - val_acc: 0.8834 - val_weighted_accuracy: 0.8746\n",
      "Epoch 13/500\n",
      " - 1s - loss: 0.0191 - acc: 0.8841 - weighted_accuracy: 0.8760 - val_loss: 0.2612 - val_acc: 0.8853 - val_weighted_accuracy: 0.8763\n",
      "Epoch 14/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8846 - weighted_accuracy: 0.8761 - val_loss: 0.2624 - val_acc: 0.8837 - val_weighted_accuracy: 0.8754\n",
      "Epoch 15/500\n",
      " - 1s - loss: 0.0191 - acc: 0.8842 - weighted_accuracy: 0.8758 - val_loss: 0.2689 - val_acc: 0.8807 - val_weighted_accuracy: 0.8745\n",
      "Epoch 16/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8842 - weighted_accuracy: 0.8759 - val_loss: 0.2651 - val_acc: 0.8829 - val_weighted_accuracy: 0.8743\n",
      "Epoch 17/500\n",
      " - 1s - loss: 0.0191 - acc: 0.8845 - weighted_accuracy: 0.8762 - val_loss: 0.2613 - val_acc: 0.8854 - val_weighted_accuracy: 0.8761\n",
      "Epoch 18/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8845 - weighted_accuracy: 0.8760 - val_loss: 0.2635 - val_acc: 0.8835 - val_weighted_accuracy: 0.8754\n",
      "Epoch 19/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8846 - weighted_accuracy: 0.8761 - val_loss: 0.2685 - val_acc: 0.8827 - val_weighted_accuracy: 0.8742\n",
      "Epoch 20/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8845 - weighted_accuracy: 0.8760 - val_loss: 0.2647 - val_acc: 0.8826 - val_weighted_accuracy: 0.8753\n",
      "Epoch 21/500\n",
      " - 1s - loss: 0.0191 - acc: 0.8846 - weighted_accuracy: 0.8762 - val_loss: 0.2599 - val_acc: 0.8850 - val_weighted_accuracy: 0.8755\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 18)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_163 (Dense)               (None, 24)           456         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_136 (Dropout)           (None, 24)           0           dense_163[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_136 (Concatenate)   (None, 42)           0           mata-features[0][0]              \n",
      "                                                                 dropout_136[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_164 (Dense)               (None, 24)           1032        concatenate_136[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_137 (Dropout)           (None, 24)           0           dense_164[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_137 (Concatenate)   (None, 66)           0           concatenate_136[0][0]            \n",
      "                                                                 dropout_137[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_165 (Dense)               (None, 24)           1608        concatenate_137[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_138 (Dropout)           (None, 24)           0           dense_165[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_138 (Concatenate)   (None, 90)           0           concatenate_137[0][0]            \n",
      "                                                                 dropout_138[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_166 (Dense)               (None, 24)           2184        concatenate_138[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_139 (Dropout)           (None, 24)           0           dense_166[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_139 (Concatenate)   (None, 114)          0           concatenate_138[0][0]            \n",
      "                                                                 dropout_139[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_167 (Dense)               (None, 24)           2760        concatenate_139[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_140 (Dropout)           (None, 24)           0           dense_167[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_140 (Concatenate)   (None, 138)          0           concatenate_139[0][0]            \n",
      "                                                                 dropout_140[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "highway_28 (Highway)            (None, 138)          38364       concatenate_140[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_168 (Dense)               (None, 3)            417         highway_28[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 46,821\n",
      "Trainable params: 46,821\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 3s - loss: 0.0215 - acc: 0.8805 - weighted_accuracy: 0.8688 - val_loss: 0.2776 - val_acc: 0.8777 - val_weighted_accuracy: 0.8709\n",
      "Epoch 2/500\n",
      " - 2s - loss: 0.0193 - acc: 0.8845 - weighted_accuracy: 0.8756 - val_loss: 0.2838 - val_acc: 0.8758 - val_weighted_accuracy: 0.8711\n",
      "Epoch 3/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8850 - weighted_accuracy: 0.8760 - val_loss: 0.2809 - val_acc: 0.8757 - val_weighted_accuracy: 0.8708\n",
      "Epoch 4/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8849 - weighted_accuracy: 0.8762 - val_loss: 0.2830 - val_acc: 0.8758 - val_weighted_accuracy: 0.8712\n",
      "Epoch 5/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8848 - weighted_accuracy: 0.8762 - val_loss: 0.2793 - val_acc: 0.8753 - val_weighted_accuracy: 0.8704\n",
      "Epoch 6/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8852 - weighted_accuracy: 0.8765 - val_loss: 0.2780 - val_acc: 0.8766 - val_weighted_accuracy: 0.8706\n",
      "Epoch 7/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8850 - weighted_accuracy: 0.8762 - val_loss: 0.2794 - val_acc: 0.8764 - val_weighted_accuracy: 0.8709\n",
      "Epoch 8/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8849 - weighted_accuracy: 0.8760 - val_loss: 0.2844 - val_acc: 0.8742 - val_weighted_accuracy: 0.8708\n",
      "Epoch 9/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8852 - weighted_accuracy: 0.8765 - val_loss: 0.2754 - val_acc: 0.8781 - val_weighted_accuracy: 0.8704\n",
      "Epoch 10/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8851 - weighted_accuracy: 0.8764 - val_loss: 0.2755 - val_acc: 0.8768 - val_weighted_accuracy: 0.8707\n",
      "Epoch 11/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8851 - weighted_accuracy: 0.8765 - val_loss: 0.2774 - val_acc: 0.8783 - val_weighted_accuracy: 0.8706\n",
      "Epoch 12/500\n",
      " - 2s - loss: 0.0189 - acc: 0.8847 - weighted_accuracy: 0.8762 - val_loss: 0.2790 - val_acc: 0.8765 - val_weighted_accuracy: 0.8705\n",
      "Epoch 13/500\n",
      " - 2s - loss: 0.0189 - acc: 0.8854 - weighted_accuracy: 0.8767 - val_loss: 0.2801 - val_acc: 0.8753 - val_weighted_accuracy: 0.8713\n",
      "Epoch 14/500\n",
      " - 2s - loss: 0.0189 - acc: 0.8854 - weighted_accuracy: 0.8766 - val_loss: 0.2810 - val_acc: 0.8752 - val_weighted_accuracy: 0.8710\n",
      "Epoch 15/500\n",
      " - 2s - loss: 0.0189 - acc: 0.8850 - weighted_accuracy: 0.8765 - val_loss: 0.2827 - val_acc: 0.8760 - val_weighted_accuracy: 0.8711\n",
      "Epoch 16/500\n",
      " - 2s - loss: 0.0189 - acc: 0.8851 - weighted_accuracy: 0.8763 - val_loss: 0.2798 - val_acc: 0.8757 - val_weighted_accuracy: 0.8711\n",
      "Epoch 17/500\n",
      " - 2s - loss: 0.0189 - acc: 0.8852 - weighted_accuracy: 0.8767 - val_loss: 0.2812 - val_acc: 0.8755 - val_weighted_accuracy: 0.8712\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18/500\n",
      " - 2s - loss: 0.0189 - acc: 0.8855 - weighted_accuracy: 0.8768 - val_loss: 0.2780 - val_acc: 0.8759 - val_weighted_accuracy: 0.8704\n",
      "Epoch 19/500\n",
      " - 2s - loss: 0.0189 - acc: 0.8854 - weighted_accuracy: 0.8768 - val_loss: 0.2885 - val_acc: 0.8703 - val_weighted_accuracy: 0.8682\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 18)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_169 (Dense)               (None, 24)           456         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_141 (Dropout)           (None, 24)           0           dense_169[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_141 (Concatenate)   (None, 42)           0           mata-features[0][0]              \n",
      "                                                                 dropout_141[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_170 (Dense)               (None, 24)           1032        concatenate_141[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_142 (Dropout)           (None, 24)           0           dense_170[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_142 (Concatenate)   (None, 66)           0           concatenate_141[0][0]            \n",
      "                                                                 dropout_142[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_171 (Dense)               (None, 24)           1608        concatenate_142[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_143 (Dropout)           (None, 24)           0           dense_171[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_143 (Concatenate)   (None, 90)           0           concatenate_142[0][0]            \n",
      "                                                                 dropout_143[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_172 (Dense)               (None, 24)           2184        concatenate_143[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_144 (Dropout)           (None, 24)           0           dense_172[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_144 (Concatenate)   (None, 114)          0           concatenate_143[0][0]            \n",
      "                                                                 dropout_144[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_173 (Dense)               (None, 24)           2760        concatenate_144[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_145 (Dropout)           (None, 24)           0           dense_173[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_145 (Concatenate)   (None, 138)          0           concatenate_144[0][0]            \n",
      "                                                                 dropout_145[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "highway_29 (Highway)            (None, 138)          38364       concatenate_145[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_174 (Dense)               (None, 3)            417         highway_29[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 46,821\n",
      "Trainable params: 46,821\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 4s - loss: 0.0215 - acc: 0.8796 - weighted_accuracy: 0.8693 - val_loss: 0.2509 - val_acc: 0.8917 - val_weighted_accuracy: 0.8859\n",
      "Epoch 2/500\n",
      " - 2s - loss: 0.0196 - acc: 0.8821 - weighted_accuracy: 0.8739 - val_loss: 0.2572 - val_acc: 0.8950 - val_weighted_accuracy: 0.8895\n",
      "Epoch 3/500\n",
      " - 2s - loss: 0.0195 - acc: 0.8826 - weighted_accuracy: 0.8739 - val_loss: 0.2648 - val_acc: 0.8921 - val_weighted_accuracy: 0.8870\n",
      "Epoch 4/500\n",
      " - 2s - loss: 0.0194 - acc: 0.8825 - weighted_accuracy: 0.8741 - val_loss: 0.2522 - val_acc: 0.8937 - val_weighted_accuracy: 0.8880\n",
      "Epoch 5/500\n",
      " - 2s - loss: 0.0194 - acc: 0.8830 - weighted_accuracy: 0.8744 - val_loss: 0.2497 - val_acc: 0.8936 - val_weighted_accuracy: 0.8879\n",
      "Epoch 6/500\n",
      " - 2s - loss: 0.0194 - acc: 0.8828 - weighted_accuracy: 0.8742 - val_loss: 0.2502 - val_acc: 0.8941 - val_weighted_accuracy: 0.8882\n",
      "Epoch 7/500\n",
      " - 2s - loss: 0.0193 - acc: 0.8830 - weighted_accuracy: 0.8745 - val_loss: 0.2522 - val_acc: 0.8946 - val_weighted_accuracy: 0.8885\n",
      "Epoch 8/500\n",
      " - 2s - loss: 0.0193 - acc: 0.8831 - weighted_accuracy: 0.8744 - val_loss: 0.2504 - val_acc: 0.8951 - val_weighted_accuracy: 0.8885\n",
      "Epoch 9/500\n",
      " - 2s - loss: 0.0193 - acc: 0.8829 - weighted_accuracy: 0.8742 - val_loss: 0.2517 - val_acc: 0.8916 - val_weighted_accuracy: 0.8869\n",
      "Epoch 10/500\n",
      " - 2s - loss: 0.0193 - acc: 0.8830 - weighted_accuracy: 0.8745 - val_loss: 0.2474 - val_acc: 0.8944 - val_weighted_accuracy: 0.8882\n",
      "Epoch 11/500\n",
      " - 2s - loss: 0.0193 - acc: 0.8829 - weighted_accuracy: 0.8744 - val_loss: 0.2513 - val_acc: 0.8937 - val_weighted_accuracy: 0.8881\n",
      "Epoch 12/500\n",
      " - 2s - loss: 0.0193 - acc: 0.8830 - weighted_accuracy: 0.8745 - val_loss: 0.2506 - val_acc: 0.8946 - val_weighted_accuracy: 0.8881\n",
      "Epoch 13/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8830 - weighted_accuracy: 0.8745 - val_loss: 0.2541 - val_acc: 0.8937 - val_weighted_accuracy: 0.8876\n",
      "Epoch 14/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8829 - weighted_accuracy: 0.8746 - val_loss: 0.2522 - val_acc: 0.8930 - val_weighted_accuracy: 0.8888\n",
      "Epoch 15/500\n",
      " - 2s - loss: 0.0193 - acc: 0.8830 - weighted_accuracy: 0.8746 - val_loss: 0.2482 - val_acc: 0.8956 - val_weighted_accuracy: 0.8891\n",
      "Epoch 16/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8831 - weighted_accuracy: 0.8747 - val_loss: 0.2551 - val_acc: 0.8937 - val_weighted_accuracy: 0.8894\n",
      "Epoch 17/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8829 - weighted_accuracy: 0.8747 - val_loss: 0.2454 - val_acc: 0.8972 - val_weighted_accuracy: 0.8899\n",
      "Epoch 18/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8831 - weighted_accuracy: 0.8745 - val_loss: 0.2464 - val_acc: 0.8957 - val_weighted_accuracy: 0.8886\n",
      "Epoch 19/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8832 - weighted_accuracy: 0.8748 - val_loss: 0.2444 - val_acc: 0.8964 - val_weighted_accuracy: 0.8891\n",
      "Epoch 20/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8832 - weighted_accuracy: 0.8747 - val_loss: 0.2484 - val_acc: 0.8936 - val_weighted_accuracy: 0.8873\n",
      "Epoch 21/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8830 - weighted_accuracy: 0.8746 - val_loss: 0.2498 - val_acc: 0.8953 - val_weighted_accuracy: 0.8894\n",
      "Epoch 22/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8833 - weighted_accuracy: 0.8747 - val_loss: 0.2439 - val_acc: 0.8963 - val_weighted_accuracy: 0.8888\n",
      "Epoch 23/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8832 - weighted_accuracy: 0.8748 - val_loss: 0.2502 - val_acc: 0.8947 - val_weighted_accuracy: 0.8888\n",
      "Epoch 24/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8833 - weighted_accuracy: 0.8748 - val_loss: 0.2499 - val_acc: 0.8936 - val_weighted_accuracy: 0.8882\n",
      "Epoch 25/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8831 - weighted_accuracy: 0.8747 - val_loss: 0.2491 - val_acc: 0.8943 - val_weighted_accuracy: 0.8882\n",
      "Epoch 26/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8830 - weighted_accuracy: 0.8747 - val_loss: 0.2445 - val_acc: 0.8971 - val_weighted_accuracy: 0.8893\n",
      "Epoch 27/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8832 - weighted_accuracy: 0.8747 - val_loss: 0.2493 - val_acc: 0.8947 - val_weighted_accuracy: 0.8884\n",
      "Epoch 28/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8831 - weighted_accuracy: 0.8749 - val_loss: 0.2494 - val_acc: 0.8939 - val_weighted_accuracy: 0.8880\n",
      "Epoch 29/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8832 - weighted_accuracy: 0.8747 - val_loss: 0.2497 - val_acc: 0.8955 - val_weighted_accuracy: 0.8891\n",
      "Epoch 30/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8830 - weighted_accuracy: 0.8746 - val_loss: 0.2474 - val_acc: 0.8947 - val_weighted_accuracy: 0.8881\n",
      "Epoch 31/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8832 - weighted_accuracy: 0.8748 - val_loss: 0.2490 - val_acc: 0.8939 - val_weighted_accuracy: 0.8883\n",
      "Epoch 32/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8832 - weighted_accuracy: 0.8748 - val_loss: 0.2492 - val_acc: 0.8944 - val_weighted_accuracy: 0.8887\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 18)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_175 (Dense)               (None, 24)           456         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_146 (Dropout)           (None, 24)           0           dense_175[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_146 (Concatenate)   (None, 42)           0           mata-features[0][0]              \n",
      "                                                                 dropout_146[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_176 (Dense)               (None, 24)           1032        concatenate_146[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_147 (Dropout)           (None, 24)           0           dense_176[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_147 (Concatenate)   (None, 66)           0           concatenate_146[0][0]            \n",
      "                                                                 dropout_147[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_177 (Dense)               (None, 24)           1608        concatenate_147[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_148 (Dropout)           (None, 24)           0           dense_177[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_148 (Concatenate)   (None, 90)           0           concatenate_147[0][0]            \n",
      "                                                                 dropout_148[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_178 (Dense)               (None, 24)           2184        concatenate_148[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_149 (Dropout)           (None, 24)           0           dense_178[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_149 (Concatenate)   (None, 114)          0           concatenate_148[0][0]            \n",
      "                                                                 dropout_149[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_179 (Dense)               (None, 24)           2760        concatenate_149[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_150 (Dropout)           (None, 24)           0           dense_179[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_150 (Concatenate)   (None, 138)          0           concatenate_149[0][0]            \n",
      "                                                                 dropout_150[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "highway_30 (Highway)            (None, 138)          38364       concatenate_150[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_180 (Dense)               (None, 3)            417         highway_30[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 46,821\n",
      "Trainable params: 46,821\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 4s - loss: 0.0216 - acc: 0.8783 - weighted_accuracy: 0.8673 - val_loss: 0.2752 - val_acc: 0.8795 - val_weighted_accuracy: 0.8752\n",
      "Epoch 2/500\n",
      " - 2s - loss: 0.0194 - acc: 0.8845 - weighted_accuracy: 0.8755 - val_loss: 0.2681 - val_acc: 0.8807 - val_weighted_accuracy: 0.8754\n",
      "Epoch 3/500\n",
      " - 2s - loss: 0.0193 - acc: 0.8843 - weighted_accuracy: 0.8753 - val_loss: 0.2733 - val_acc: 0.8780 - val_weighted_accuracy: 0.8744\n",
      "Epoch 4/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8842 - weighted_accuracy: 0.8754 - val_loss: 0.2742 - val_acc: 0.8784 - val_weighted_accuracy: 0.8747\n",
      "Epoch 5/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8841 - weighted_accuracy: 0.8756 - val_loss: 0.2760 - val_acc: 0.8791 - val_weighted_accuracy: 0.8751\n",
      "Epoch 6/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8843 - weighted_accuracy: 0.8758 - val_loss: 0.2716 - val_acc: 0.8795 - val_weighted_accuracy: 0.8753\n",
      "Epoch 7/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8844 - weighted_accuracy: 0.8759 - val_loss: 0.2680 - val_acc: 0.8822 - val_weighted_accuracy: 0.8744\n",
      "Epoch 8/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8846 - weighted_accuracy: 0.8758 - val_loss: 0.2711 - val_acc: 0.8792 - val_weighted_accuracy: 0.8751\n",
      "Epoch 9/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8844 - weighted_accuracy: 0.8757 - val_loss: 0.2669 - val_acc: 0.8811 - val_weighted_accuracy: 0.8757\n",
      "Epoch 10/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8844 - weighted_accuracy: 0.8760 - val_loss: 0.2706 - val_acc: 0.8801 - val_weighted_accuracy: 0.8753\n",
      "Epoch 11/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8845 - weighted_accuracy: 0.8758 - val_loss: 0.2682 - val_acc: 0.8812 - val_weighted_accuracy: 0.8759\n",
      "Epoch 12/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8844 - weighted_accuracy: 0.8758 - val_loss: 0.2666 - val_acc: 0.8817 - val_weighted_accuracy: 0.8759\n",
      "Epoch 13/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8849 - weighted_accuracy: 0.8761 - val_loss: 0.2671 - val_acc: 0.8806 - val_weighted_accuracy: 0.8753\n",
      "Epoch 14/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8845 - weighted_accuracy: 0.8759 - val_loss: 0.2672 - val_acc: 0.8808 - val_weighted_accuracy: 0.8756\n",
      "Epoch 15/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8846 - weighted_accuracy: 0.8761 - val_loss: 0.2728 - val_acc: 0.8800 - val_weighted_accuracy: 0.8750\n",
      "Epoch 16/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8846 - weighted_accuracy: 0.8759 - val_loss: 0.2705 - val_acc: 0.8804 - val_weighted_accuracy: 0.8753\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8850 - weighted_accuracy: 0.8762 - val_loss: 0.2701 - val_acc: 0.8805 - val_weighted_accuracy: 0.8754\n",
      "Epoch 18/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8849 - weighted_accuracy: 0.8760 - val_loss: 0.2716 - val_acc: 0.8802 - val_weighted_accuracy: 0.8752\n",
      "Epoch 19/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8847 - weighted_accuracy: 0.8760 - val_loss: 0.2690 - val_acc: 0.8810 - val_weighted_accuracy: 0.8758\n",
      "Epoch 20/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8847 - weighted_accuracy: 0.8761 - val_loss: 0.2712 - val_acc: 0.8792 - val_weighted_accuracy: 0.8753\n",
      "Epoch 21/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8848 - weighted_accuracy: 0.8763 - val_loss: 0.2661 - val_acc: 0.8817 - val_weighted_accuracy: 0.8755\n",
      "Epoch 22/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8849 - weighted_accuracy: 0.8764 - val_loss: 0.2662 - val_acc: 0.8820 - val_weighted_accuracy: 0.8747\n",
      "Epoch 23/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8847 - weighted_accuracy: 0.8760 - val_loss: 0.2672 - val_acc: 0.8809 - val_weighted_accuracy: 0.8759\n",
      "Epoch 24/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8844 - weighted_accuracy: 0.8760 - val_loss: 0.2695 - val_acc: 0.8811 - val_weighted_accuracy: 0.8752\n",
      "Epoch 25/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8846 - weighted_accuracy: 0.8760 - val_loss: 0.2695 - val_acc: 0.8814 - val_weighted_accuracy: 0.8753\n",
      "Epoch 26/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8850 - weighted_accuracy: 0.8763 - val_loss: 0.2716 - val_acc: 0.8791 - val_weighted_accuracy: 0.8751\n",
      "Epoch 27/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8846 - weighted_accuracy: 0.8761 - val_loss: 0.2703 - val_acc: 0.8803 - val_weighted_accuracy: 0.8751\n",
      "Epoch 28/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8849 - weighted_accuracy: 0.8762 - val_loss: 0.2737 - val_acc: 0.8800 - val_weighted_accuracy: 0.8755\n",
      "Epoch 29/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8845 - weighted_accuracy: 0.8760 - val_loss: 0.2673 - val_acc: 0.8811 - val_weighted_accuracy: 0.8751\n",
      "Epoch 30/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8848 - weighted_accuracy: 0.8763 - val_loss: 0.2668 - val_acc: 0.8820 - val_weighted_accuracy: 0.8753\n",
      "Epoch 31/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8848 - weighted_accuracy: 0.8762 - val_loss: 0.2684 - val_acc: 0.8815 - val_weighted_accuracy: 0.8751\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 18)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_181 (Dense)               (None, 24)           456         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_151 (Dropout)           (None, 24)           0           dense_181[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_151 (Concatenate)   (None, 42)           0           mata-features[0][0]              \n",
      "                                                                 dropout_151[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_182 (Dense)               (None, 24)           1032        concatenate_151[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_152 (Dropout)           (None, 24)           0           dense_182[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_152 (Concatenate)   (None, 66)           0           concatenate_151[0][0]            \n",
      "                                                                 dropout_152[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_183 (Dense)               (None, 24)           1608        concatenate_152[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_153 (Dropout)           (None, 24)           0           dense_183[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_153 (Concatenate)   (None, 90)           0           concatenate_152[0][0]            \n",
      "                                                                 dropout_153[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_184 (Dense)               (None, 24)           2184        concatenate_153[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_154 (Dropout)           (None, 24)           0           dense_184[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_154 (Concatenate)   (None, 114)          0           concatenate_153[0][0]            \n",
      "                                                                 dropout_154[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_185 (Dense)               (None, 24)           2760        concatenate_154[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_155 (Dropout)           (None, 24)           0           dense_185[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_155 (Concatenate)   (None, 138)          0           concatenate_154[0][0]            \n",
      "                                                                 dropout_155[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "highway_31 (Highway)            (None, 138)          38364       concatenate_155[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_186 (Dense)               (None, 3)            417         highway_31[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 46,821\n",
      "Trainable params: 46,821\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 4s - loss: 0.0211 - acc: 0.8817 - weighted_accuracy: 0.8713 - val_loss: 0.2802 - val_acc: 0.8745 - val_weighted_accuracy: 0.8598\n",
      "Epoch 2/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8859 - weighted_accuracy: 0.8769 - val_loss: 0.2785 - val_acc: 0.8744 - val_weighted_accuracy: 0.8587\n",
      "Epoch 3/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8863 - weighted_accuracy: 0.8774 - val_loss: 0.2835 - val_acc: 0.8729 - val_weighted_accuracy: 0.8606\n",
      "Epoch 4/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8861 - weighted_accuracy: 0.8771 - val_loss: 0.2792 - val_acc: 0.8740 - val_weighted_accuracy: 0.8602\n",
      "Epoch 5/500\n",
      " - 2s - loss: 0.0189 - acc: 0.8863 - weighted_accuracy: 0.8775 - val_loss: 0.2775 - val_acc: 0.8741 - val_weighted_accuracy: 0.8589\n",
      "Epoch 6/500\n",
      " - 2s - loss: 0.0189 - acc: 0.8863 - weighted_accuracy: 0.8776 - val_loss: 0.2797 - val_acc: 0.8733 - val_weighted_accuracy: 0.8613\n",
      "Epoch 7/500\n",
      " - 2s - loss: 0.0189 - acc: 0.8861 - weighted_accuracy: 0.8772 - val_loss: 0.2787 - val_acc: 0.8740 - val_weighted_accuracy: 0.8592\n",
      "Epoch 8/500\n",
      " - 2s - loss: 0.0189 - acc: 0.8864 - weighted_accuracy: 0.8776 - val_loss: 0.2809 - val_acc: 0.8724 - val_weighted_accuracy: 0.8611\n",
      "Epoch 9/500\n",
      " - 2s - loss: 0.0189 - acc: 0.8865 - weighted_accuracy: 0.8779 - val_loss: 0.2835 - val_acc: 0.8728 - val_weighted_accuracy: 0.8611\n",
      "Epoch 10/500\n",
      " - 2s - loss: 0.0188 - acc: 0.8866 - weighted_accuracy: 0.8776 - val_loss: 0.2835 - val_acc: 0.8719 - val_weighted_accuracy: 0.8629\n",
      "Epoch 11/500\n",
      " - 2s - loss: 0.0189 - acc: 0.8860 - weighted_accuracy: 0.8774 - val_loss: 0.2777 - val_acc: 0.8742 - val_weighted_accuracy: 0.8595\n",
      "Epoch 12/500\n",
      " - 2s - loss: 0.0188 - acc: 0.8864 - weighted_accuracy: 0.8776 - val_loss: 0.2820 - val_acc: 0.8740 - val_weighted_accuracy: 0.8603\n",
      "Epoch 13/500\n",
      " - 2s - loss: 0.0188 - acc: 0.8865 - weighted_accuracy: 0.8777 - val_loss: 0.2786 - val_acc: 0.8732 - val_weighted_accuracy: 0.8599\n",
      "Epoch 14/500\n",
      " - 2s - loss: 0.0188 - acc: 0.8865 - weighted_accuracy: 0.8777 - val_loss: 0.2823 - val_acc: 0.8726 - val_weighted_accuracy: 0.8618\n",
      "Epoch 15/500\n",
      " - 2s - loss: 0.0188 - acc: 0.8869 - weighted_accuracy: 0.8782 - val_loss: 0.2796 - val_acc: 0.8740 - val_weighted_accuracy: 0.8614\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 18)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_187 (Dense)               (None, 24)           456         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_156 (Dropout)           (None, 24)           0           dense_187[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_156 (Concatenate)   (None, 42)           0           mata-features[0][0]              \n",
      "                                                                 dropout_156[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_188 (Dense)               (None, 24)           1032        concatenate_156[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_157 (Dropout)           (None, 24)           0           dense_188[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_157 (Concatenate)   (None, 66)           0           concatenate_156[0][0]            \n",
      "                                                                 dropout_157[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_189 (Dense)               (None, 24)           1608        concatenate_157[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_158 (Dropout)           (None, 24)           0           dense_189[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_158 (Concatenate)   (None, 90)           0           concatenate_157[0][0]            \n",
      "                                                                 dropout_158[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_190 (Dense)               (None, 24)           2184        concatenate_158[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_159 (Dropout)           (None, 24)           0           dense_190[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_159 (Concatenate)   (None, 114)          0           concatenate_158[0][0]            \n",
      "                                                                 dropout_159[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_191 (Dense)               (None, 24)           2760        concatenate_159[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_160 (Dropout)           (None, 24)           0           dense_191[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_160 (Concatenate)   (None, 138)          0           concatenate_159[0][0]            \n",
      "                                                                 dropout_160[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "highway_32 (Highway)            (None, 138)          38364       concatenate_160[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_192 (Dense)               (None, 3)            417         highway_32[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 46,821\n",
      "Trainable params: 46,821\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 4s - loss: 0.0216 - acc: 0.8800 - weighted_accuracy: 0.8686 - val_loss: 0.2789 - val_acc: 0.8767 - val_weighted_accuracy: 0.8711\n",
      "Epoch 2/500\n",
      " - 2s - loss: 0.0195 - acc: 0.8844 - weighted_accuracy: 0.8757 - val_loss: 0.2737 - val_acc: 0.8784 - val_weighted_accuracy: 0.8715\n",
      "Epoch 3/500\n",
      " - 2s - loss: 0.0193 - acc: 0.8845 - weighted_accuracy: 0.8760 - val_loss: 0.2689 - val_acc: 0.8812 - val_weighted_accuracy: 0.8723\n",
      "Epoch 4/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8846 - weighted_accuracy: 0.8761 - val_loss: 0.2696 - val_acc: 0.8799 - val_weighted_accuracy: 0.8716\n",
      "Epoch 5/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8848 - weighted_accuracy: 0.8763 - val_loss: 0.2734 - val_acc: 0.8776 - val_weighted_accuracy: 0.8710\n",
      "Epoch 6/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8847 - weighted_accuracy: 0.8764 - val_loss: 0.2714 - val_acc: 0.8786 - val_weighted_accuracy: 0.8707\n",
      "Epoch 7/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8844 - weighted_accuracy: 0.8761 - val_loss: 0.2695 - val_acc: 0.8790 - val_weighted_accuracy: 0.8709\n",
      "Epoch 8/500\n",
      " - 1s - loss: 0.0191 - acc: 0.8848 - weighted_accuracy: 0.8764 - val_loss: 0.2690 - val_acc: 0.8795 - val_weighted_accuracy: 0.8719\n",
      "Epoch 9/500\n",
      " - 1s - loss: 0.0191 - acc: 0.8847 - weighted_accuracy: 0.8764 - val_loss: 0.2765 - val_acc: 0.8763 - val_weighted_accuracy: 0.8703\n",
      "Epoch 10/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8845 - weighted_accuracy: 0.8762 - val_loss: 0.2733 - val_acc: 0.8773 - val_weighted_accuracy: 0.8710\n",
      "Epoch 11/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8848 - weighted_accuracy: 0.8765 - val_loss: 0.2754 - val_acc: 0.8765 - val_weighted_accuracy: 0.8705\n",
      "Epoch 12/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8850 - weighted_accuracy: 0.8765 - val_loss: 0.2691 - val_acc: 0.8803 - val_weighted_accuracy: 0.8715\n",
      "Epoch 13/500\n",
      " - 2s - loss: 0.0190 - acc: 0.8849 - weighted_accuracy: 0.8765 - val_loss: 0.2772 - val_acc: 0.8765 - val_weighted_accuracy: 0.8705\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 18)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_193 (Dense)               (None, 24)           456         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_161 (Dropout)           (None, 24)           0           dense_193[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_161 (Concatenate)   (None, 42)           0           mata-features[0][0]              \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                                 dropout_161[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_194 (Dense)               (None, 24)           1032        concatenate_161[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_162 (Dropout)           (None, 24)           0           dense_194[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_162 (Concatenate)   (None, 66)           0           concatenate_161[0][0]            \n",
      "                                                                 dropout_162[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_195 (Dense)               (None, 24)           1608        concatenate_162[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_163 (Dropout)           (None, 24)           0           dense_195[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_163 (Concatenate)   (None, 90)           0           concatenate_162[0][0]            \n",
      "                                                                 dropout_163[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_196 (Dense)               (None, 24)           2184        concatenate_163[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_164 (Dropout)           (None, 24)           0           dense_196[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_164 (Concatenate)   (None, 114)          0           concatenate_163[0][0]            \n",
      "                                                                 dropout_164[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_197 (Dense)               (None, 24)           2760        concatenate_164[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_165 (Dropout)           (None, 24)           0           dense_197[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_165 (Concatenate)   (None, 138)          0           concatenate_164[0][0]            \n",
      "                                                                 dropout_165[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "highway_33 (Highway)            (None, 138)          38364       concatenate_165[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_198 (Dense)               (None, 3)            417         highway_33[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 46,821\n",
      "Trainable params: 46,821\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288497 samples, validate on 32055 samples\n",
      "Epoch 1/500\n",
      " - 4s - loss: 0.0219 - acc: 0.8756 - weighted_accuracy: 0.8649 - val_loss: 0.2629 - val_acc: 0.8879 - val_weighted_accuracy: 0.8806\n",
      "Epoch 2/500\n",
      " - 2s - loss: 0.0195 - acc: 0.8833 - weighted_accuracy: 0.8745 - val_loss: 0.2648 - val_acc: 0.8869 - val_weighted_accuracy: 0.8813\n",
      "Epoch 3/500\n",
      " - 2s - loss: 0.0193 - acc: 0.8836 - weighted_accuracy: 0.8752 - val_loss: 0.2601 - val_acc: 0.8873 - val_weighted_accuracy: 0.8811\n",
      "Epoch 4/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8834 - weighted_accuracy: 0.8752 - val_loss: 0.2596 - val_acc: 0.8888 - val_weighted_accuracy: 0.8814\n",
      "Epoch 5/500\n",
      " - 1s - loss: 0.0193 - acc: 0.8837 - weighted_accuracy: 0.8750 - val_loss: 0.2591 - val_acc: 0.8895 - val_weighted_accuracy: 0.8817\n",
      "Epoch 6/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8838 - weighted_accuracy: 0.8753 - val_loss: 0.2633 - val_acc: 0.8866 - val_weighted_accuracy: 0.8808\n",
      "Epoch 7/500\n",
      " - 1s - loss: 0.0192 - acc: 0.8835 - weighted_accuracy: 0.8753 - val_loss: 0.2605 - val_acc: 0.8882 - val_weighted_accuracy: 0.8815\n",
      "Epoch 8/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8839 - weighted_accuracy: 0.8755 - val_loss: 0.2608 - val_acc: 0.8864 - val_weighted_accuracy: 0.8808\n",
      "Epoch 9/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8838 - weighted_accuracy: 0.8754 - val_loss: 0.2622 - val_acc: 0.8869 - val_weighted_accuracy: 0.8811\n",
      "Epoch 10/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8840 - weighted_accuracy: 0.8756 - val_loss: 0.2580 - val_acc: 0.8888 - val_weighted_accuracy: 0.8813\n",
      "Epoch 11/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8838 - weighted_accuracy: 0.8752 - val_loss: 0.2649 - val_acc: 0.8857 - val_weighted_accuracy: 0.8807\n",
      "Epoch 12/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8837 - weighted_accuracy: 0.8754 - val_loss: 0.2604 - val_acc: 0.8881 - val_weighted_accuracy: 0.8814\n",
      "Epoch 13/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8839 - weighted_accuracy: 0.8754 - val_loss: 0.2622 - val_acc: 0.8867 - val_weighted_accuracy: 0.8809\n",
      "Epoch 14/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8836 - weighted_accuracy: 0.8753 - val_loss: 0.2589 - val_acc: 0.8883 - val_weighted_accuracy: 0.8807\n",
      "Epoch 15/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8841 - weighted_accuracy: 0.8755 - val_loss: 0.2614 - val_acc: 0.8875 - val_weighted_accuracy: 0.8813\n",
      "Epoch 16/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8837 - weighted_accuracy: 0.8753 - val_loss: 0.2616 - val_acc: 0.8863 - val_weighted_accuracy: 0.8808\n",
      "Epoch 17/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8840 - weighted_accuracy: 0.8756 - val_loss: 0.2615 - val_acc: 0.8872 - val_weighted_accuracy: 0.8810\n",
      "Epoch 18/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8840 - weighted_accuracy: 0.8755 - val_loss: 0.2603 - val_acc: 0.8882 - val_weighted_accuracy: 0.8816\n",
      "Epoch 19/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8842 - weighted_accuracy: 0.8756 - val_loss: 0.2625 - val_acc: 0.8870 - val_weighted_accuracy: 0.8811\n",
      "Epoch 20/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8839 - weighted_accuracy: 0.8756 - val_loss: 0.2589 - val_acc: 0.8886 - val_weighted_accuracy: 0.8812\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 18)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_199 (Dense)               (None, 24)           456         mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_166 (Dropout)           (None, 24)           0           dense_199[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_166 (Concatenate)   (None, 42)           0           mata-features[0][0]              \n",
      "                                                                 dropout_166[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_200 (Dense)               (None, 24)           1032        concatenate_166[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_167 (Dropout)           (None, 24)           0           dense_200[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_167 (Concatenate)   (None, 66)           0           concatenate_166[0][0]            \n",
      "                                                                 dropout_167[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_201 (Dense)               (None, 24)           1608        concatenate_167[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_168 (Dropout)           (None, 24)           0           dense_201[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_168 (Concatenate)   (None, 90)           0           concatenate_167[0][0]            \n",
      "                                                                 dropout_168[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_202 (Dense)               (None, 24)           2184        concatenate_168[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_169 (Dropout)           (None, 24)           0           dense_202[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_169 (Concatenate)   (None, 114)          0           concatenate_168[0][0]            \n",
      "                                                                 dropout_169[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_203 (Dense)               (None, 24)           2760        concatenate_169[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_170 (Dropout)           (None, 24)           0           dense_203[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_170 (Concatenate)   (None, 138)          0           concatenate_169[0][0]            \n",
      "                                                                 dropout_170[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "highway_34 (Highway)            (None, 138)          38364       concatenate_170[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_204 (Dense)               (None, 3)            417         highway_34[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 46,821\n",
      "Trainable params: 46,821\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Train on 288495 samples, validate on 32057 samples\n",
      "Epoch 1/500\n",
      " - 4s - loss: 0.0213 - acc: 0.8806 - weighted_accuracy: 0.8707 - val_loss: 0.2619 - val_acc: 0.8867 - val_weighted_accuracy: 0.8766\n",
      "Epoch 2/500\n",
      " - 2s - loss: 0.0194 - acc: 0.8834 - weighted_accuracy: 0.8751 - val_loss: 0.2542 - val_acc: 0.8889 - val_weighted_accuracy: 0.8772\n",
      "Epoch 3/500\n",
      " - 2s - loss: 0.0193 - acc: 0.8839 - weighted_accuracy: 0.8756 - val_loss: 0.2535 - val_acc: 0.8894 - val_weighted_accuracy: 0.8775\n",
      "Epoch 4/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8836 - weighted_accuracy: 0.8757 - val_loss: 0.2574 - val_acc: 0.8862 - val_weighted_accuracy: 0.8764\n",
      "Epoch 5/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8837 - weighted_accuracy: 0.8758 - val_loss: 0.2535 - val_acc: 0.8883 - val_weighted_accuracy: 0.8769\n",
      "Epoch 6/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8834 - weighted_accuracy: 0.8757 - val_loss: 0.2572 - val_acc: 0.8876 - val_weighted_accuracy: 0.8773\n",
      "Epoch 7/500\n",
      " - 2s - loss: 0.0192 - acc: 0.8834 - weighted_accuracy: 0.8755 - val_loss: 0.2574 - val_acc: 0.8874 - val_weighted_accuracy: 0.8770\n",
      "Epoch 8/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8835 - weighted_accuracy: 0.8757 - val_loss: 0.2511 - val_acc: 0.8904 - val_weighted_accuracy: 0.8776\n",
      "Epoch 9/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8835 - weighted_accuracy: 0.8758 - val_loss: 0.2583 - val_acc: 0.8862 - val_weighted_accuracy: 0.8764\n",
      "Epoch 10/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8834 - weighted_accuracy: 0.8756 - val_loss: 0.2567 - val_acc: 0.8868 - val_weighted_accuracy: 0.8766\n",
      "Epoch 11/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8834 - weighted_accuracy: 0.8756 - val_loss: 0.2542 - val_acc: 0.8896 - val_weighted_accuracy: 0.8772\n",
      "Epoch 12/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8836 - weighted_accuracy: 0.8758 - val_loss: 0.2559 - val_acc: 0.8887 - val_weighted_accuracy: 0.8771\n",
      "Epoch 13/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8836 - weighted_accuracy: 0.8759 - val_loss: 0.2528 - val_acc: 0.8905 - val_weighted_accuracy: 0.8775\n",
      "Epoch 14/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8838 - weighted_accuracy: 0.8760 - val_loss: 0.2535 - val_acc: 0.8898 - val_weighted_accuracy: 0.8779\n",
      "Epoch 15/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8839 - weighted_accuracy: 0.8760 - val_loss: 0.2528 - val_acc: 0.8891 - val_weighted_accuracy: 0.8773\n",
      "Epoch 16/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8836 - weighted_accuracy: 0.8757 - val_loss: 0.2545 - val_acc: 0.8883 - val_weighted_accuracy: 0.8769\n",
      "Epoch 17/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8839 - weighted_accuracy: 0.8760 - val_loss: 0.2537 - val_acc: 0.8899 - val_weighted_accuracy: 0.8769\n",
      "Epoch 18/500\n",
      " - 2s - loss: 0.0191 - acc: 0.8838 - weighted_accuracy: 0.8758 - val_loss: 0.2566 - val_acc: 0.8880 - val_weighted_accuracy: 0.8770\n"
     ]
    }
   ],
   "source": [
    "trainer = KerasModelTrainer(model_stamp=\"Ensemble-DenseNet\", epoch_num=500)\n",
    "models, score, folds_preds = trainer.train_folds(features=ensemble_trains, y=to_categorical(labels), augments=None, fold_count=10,\n",
    "    batch_size=1024, \n",
    "    scale_sample_weight=None, class_weight=None,\n",
    "    get_model_func=_agent_get_model, \n",
    "    patience=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8771482783241311"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score 0.8771482783241311\n",
      "Predicting training results...\n",
      "Predicting testing results...\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 11us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 11us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 10us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 10us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 10us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 10us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 10us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 10us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 10us/step\n",
      "80126/80126 [==============================] - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - ETA:  - 1s 10us/step\n",
      "Predicting labeled testing results...\n"
     ]
    }
   ],
   "source": [
    "print(\"score\", score)\n",
    "oofs_dir = \"../data/p_ensemble/second_level/oofs/\"\n",
    "output_dir = \"../data/p_ensemble/second_level/preds/\"\n",
    "onehot_pred_dir = \"../data/p_ensemble/second_level/nn_one_hot/\"\n",
    "\n",
    "model_submit_prefix = \"AddNN-Ensemble\"\n",
    "\n",
    "oofs_path = oofs_dir + model_submit_prefix\n",
    "output_path = output_dir + model_submit_prefix\n",
    "one_hot_pred_path = onehot_pred_dir + \"One-Hot\" + model_submit_prefix\n",
    "\n",
    "print(\"Predicting training results...\")\n",
    "train_predicts = np.concatenate(folds_preds, axis=0)\n",
    "score = np_weighted_accuracy(to_categorical(labels), train_predicts)\n",
    "\n",
    "oofs = pd.DataFrame({\"unrelated\": train_predicts[:, 0], \"agreed\": train_predicts[:, 1], \"disagreed\": train_predicts[:, 2]})\n",
    "submit_path = oofs_path + \"-Train-L{:4f}-NB{:d}.csv\".format(score, NB_WORDS)\n",
    "oofs.to_csv(submit_path, index=False)\n",
    "\n",
    "print(\"Predicting testing results...\")\n",
    "test_predicts_list = []\n",
    "for fold_id, model in enumerate(models):\n",
    "    test_predicts = model.predict({\"mata-features\": ensemble_tests}, batch_size=128, verbose=1)\n",
    "    test_predicts_list.append(test_predicts)\n",
    "\n",
    "test_predicts = np.zeros(test_predicts_list[0].shape)\n",
    "for fold_predict in test_predicts_list:\n",
    "    test_predicts += fold_predict\n",
    "test_predicts /= len(test_predicts_list)\n",
    "\n",
    "test_predicts = pd.DataFrame({\"unrelated\": test_predicts[:, 0], \"agreed\": test_predicts[:, 1], \"disagreed\": test_predicts[:, 2]})\n",
    "submit_path = output_path + \"-L{:4f}-NB{:d}.csv\".format(score, NB_WORDS)\n",
    "test_predicts.to_csv(submit_path, index=False) # 0.3343\n",
    "\n",
    "print(\"Predicting labeled testing results...\")\n",
    "ids = pd.read_csv(\"../data/dataset/test.csv\")\n",
    "pred_labels = test_predicts.idxmax(axis=1)\n",
    "sub = pd.DataFrame({\"Id\": ids['id'].values, \"Category\": pred_labels})\n",
    "submit_path = one_hot_pred_path + \"-L{:4f}-NB{:d}.csv\".format(score, NB_WORDS)\n",
    "sub.to_csv(submit_path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "unrelated    0.638569\n",
       "agreed       0.334261\n",
       "disagreed    0.027170\n",
       "Name: Category, dtype: float64"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub['Category'].value_counts() / len(sub)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_predicts.to_csv(\"../data/high_ground/fine_tuned_nns.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 1,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
